/*
 * Parser for Wikipedia dumps
 * 
 * @version 1.0
 * @author Till Speicher
 */
package complet.app.io.wikiparser;

import edu.jhu.nlp.wikipedia.PageCallbackHandler;
import edu.jhu.nlp.wikipedia.WikiPage;
import edu.jhu.nlp.wikipedia.WikiXMLParser;
import edu.jhu.nlp.wikipedia.WikiXMLParserFactory;
import java.util.HashMap;
import java.io.FileWriter;
import java.io.File;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class Main {

    //constants
    private static String IN_PATH = "C:\\dewiki-20111212-pages-meta-current.xml";
    private static String OUT_PATH = "E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\inputfiles\\wikipedia\\wiki_info_small_uc.txt";
    private static String STATISTICS = "E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\inputfiles\\wikipedia\\log.txt";
    private static String[] seperators = {"\\. ", ":", ";", "! ", "\\? "};
    private static String[] wikiSymbols = {"\\*", "== ", " ==", "\\|"};
    private static String[] trash = {",", "\\.", "!"};

    public static void main(String[] args) {
        WikiXMLParser wxsp = WikiXMLParserFactory.getSAXParser(IN_PATH);


        try {
            wxsp.setPageCallback(new PageCallbackHandler() {

                public void process(WikiPage page) {
                    globalStorage.articleCounter++;

                    HashMap<String, String> replacements = new HashMap<String, String>();
//                    replacements.put("ä", "ae");
//                    replacements.put("Ä", "ae");
//                    replacements.put("ö", "oe");
//                    replacements.put("Ö", "oe");
//                    replacements.put("ü", "ue");
//                    replacements.put("Ü", "ue");
//                    replacements.put("ß", "ss");
                    replacements.put(".&nbsp;", " ");
//                    replacements.put("e", "e");
                    replacements.put("-", " ");

                    // no Special Sites please
                    if(page.isSpecialPage()){
//                        return;
                    }
                    System.out.println("Processing: " + page.getTitle() + " - " + globalStorage.articleCounter);
                    String pageText = page.getText();
//                    pageText = pageText.toLowerCase();

                    for (String key : replacements.keySet()) {
                        pageText = pageText.replaceAll(key, replacements.get(key));
                    }

                    //insert word wrap
                    for (int i = 0; i < seperators.length; i++) {
                        pageText = pageText.replaceAll(seperators[i], "\n");
                    }

                    //delete numbers
                    pageText = pageText.replaceAll("[0-9]", "");

                    //delete text in brackets
                    pageText = pageText.replaceAll("\\([. \n]*\\)", "");


                    //delete wiki symbols
                    for (int i = 0; i < wikiSymbols.length; i++) {
                        pageText = pageText.replaceAll(wikiSymbols[i], "");
                    }

                    //delete single characters
                    pageText = pageText.replaceAll(" .( .)*[ \n]", " ");

                    for (int i = 0; i < trash.length; i++) {
                        pageText = pageText.replaceAll(trash[i], "");
                    }

                    //delete mulitple whitespaces
                    while (pageText.contains("  ")) {
                        pageText = pageText.replaceAll("  ", " ");
                    }

                    String[] lineIterator = pageText.split("\n");
                    for (int i = 0; i < lineIterator.length; i++) {
                        lineIterator[i] = lineIterator[i].trim();
                    }

                    //filter sentences with less than 5 words and/or with special characters
                    String result = "";
                    Pattern p = Pattern.compile("[^a-zA-ZÄÜÖäüöß\n ]");
                    for (int i = 0; i < lineIterator.length; i++) {
                        Matcher m = p.matcher(lineIterator[i]);
                        if (lineIterator[i].split(" ").length >= 5 && m.find() == false) {
                            lineIterator[i] += "\t" + globalStorage.sentenceCounter;
                            result += lineIterator[i] + "\n";
                            globalStorage.sentenceCounter++;
                        } else {
                            globalStorage.removedCounter++;
                        }
                        globalStorage.overallCounter++;
                    }

                    Main.output(result);

                    if(globalStorage.sentenceCounter >= 500000){
                        System.exit(1);
                    }

//					System.out.println(result);
                }
            });

            wxsp.parse();
        } catch (Exception e) {
            e.printStackTrace();
        }

    }

    //output to file
    public static void output(String result) {
        try {
            File file = new File(OUT_PATH);
            if (file.exists() && globalStorage.sentenceCounter == 0) {
                FileWriter out = new FileWriter(file, false);
                out.write("");
            }

            FileWriter out = new FileWriter(file, true);

            out.write(result);

            out.flush();
            out.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
        try {
            File file = new File(STATISTICS);
            FileWriter out = new FileWriter(file, false);

            out.write("Saetze in der Datei: " + globalStorage.sentenceCounter + "\n");
            out.write("Entfallene Saetze: " + globalStorage.removedCounter + "\n");
            out.write("Saetze gesamt: " + globalStorage.overallCounter + "\n");
            out.write("Artikel: " + globalStorage.articleCounter + "\n");

            out.flush();
            out.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}
